{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e98cd38a",
   "metadata": {},
   "source": [
    "# RAG loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ea93eb7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/gjf/新加卷/code/challenge_cup/MindCare-LLM\n"
     ]
    }
   ],
   "source": [
    "%cd ../"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c2319c60",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gjf/miniconda3/envs/langchain/lib/python3.11/site-packages/requests/__init__.py:86: RequestsDependencyWarning: Unable to find acceptable character detection dependency (chardet or charset_normalizer).\n",
      "  warnings.warn(\n",
      "USER_AGENT environment variable not set, consider setting it to identify your requests.\n"
     ]
    }
   ],
   "source": [
    "from rag.utils import *\n",
    "import os\n",
    "import fnmatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3719c327",
   "metadata": {},
   "outputs": [],
   "source": [
    "databaae_path = \"./data/database1\"\n",
    "pdf_docs = []\n",
    "web_docs = []\n",
    "text_docs = []"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a6ada74",
   "metadata": {},
   "source": [
    "## load docs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdb2b60d",
   "metadata": {},
   "source": [
    "### load pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "890dca6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pdf_files = []\n",
    "\n",
    "# 遍历目录及其子目录\n",
    "for root, dirs, files in os.walk(databaae_path):\n",
    "    for filename in fnmatch.filter(files, \"*.pdf\"):\n",
    "        # 将找到的PDF文件的完整路径添加到列表中\n",
    "        pdf_files.append(os.path.join(root, filename))\n",
    "\n",
    "for file_path in pdf_files:\n",
    "    pdf_docs += pdf_loader(file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5352000d",
   "metadata": {},
   "source": [
    "### load web url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe01a97a",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = [\"\"]\n",
    "web_docs = web_loader(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e1bb256",
   "metadata": {},
   "source": [
    "### load text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d197924a",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_files = []\n",
    "\n",
    "# 遍历目录及其子目录\n",
    "for root, dirs, files in os.walk(databaae_path):\n",
    "    for filename in fnmatch.filter(files, \"*.txt\"):\n",
    "        # 将找到的text文件的完整路径添加到列表中\n",
    "        text_files.append(os.path.join(root, filename))\n",
    "\n",
    "for file_path in text_files:\n",
    "    text_docs += text_loader(file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "113ae19d",
   "metadata": {},
   "source": [
    "## Text clean and split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "66661a02",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_docs = pdf_docs + web_docs + text_docs\n",
    "for doc in all_docs:\n",
    "    doc.page_content = clean_text(doc.page_content)\n",
    "splitted_docs = doc_splitter(all_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da642261",
   "metadata": {},
   "source": [
    "## Make metadata and save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "231798ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['HTTP_PROXY'] = 'http://10.22.71.69:37890' \n",
    "os.environ['HTTPS_PROXY'] = 'http://10.22.71.69:37890' \n",
    "make_metadata(splitted_docs)\n",
    "save_database(splitted_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4963027",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8c67c26e",
   "metadata": {},
   "source": [
    "## Add new docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa448930",
   "metadata": {},
   "outputs": [],
   "source": [
    "update_vector_store(splitted_docs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "langchain",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
